from bs4 import BeautifulSoup
from time import sleep
from selenium import webdriver
import re
import json


class b2b(object):
    def __init__(self, name, max_page):
        self.name = name
        self.url = "https://b2b.baidu.com/s?q={}&from=search".format(self.name)
        self.driver = webdriver.Chrome(executable_path = './chromedriver')
        self.page_num = 1
        self.max_page = max_page
        self.data = []

    def run(self):
        self.driver.get(self.url)
        print(self.page_num)
        if self.page_num == 1:
            self.driver.find_element_by_class_name("ivu-checkbox-input").click()
        self.get_data()

    def get_data(self):
        print("正在抓取"+self.name+"页面第"+str(self.page_num)+"页")
        js = "window.scrollBy(0, 3000)"

        for i in range(5):
            self.driver.execute_script(js)
            sleep(1)

        page = self.driver.page_source
        soup = BeautifulSoup(page, "lxml")
        div = soup.find("div", class_="product-list").find_all_next("div", class_="inline")
        for d in div:
            try:
                title = d.find("div", class_="p-card-img-layout").attrs["title"]
                href = d.find("a", class_="link p-card-href").attrs["href"]
                self.data.append(href)
            except:
                pass
        self.next_page()

    def next_page(self):
        print("第" + str(self.page_num) + "页爬取成功")
        try:
            if self.page_num < self.max_page:
                self.page_num += 1
                self.driver.find_element_by_class_name("ivu-page-next").click()
                sleep(1)
                self.get_data()
            else:
                self.save_data()
        except Exception as s:
            print("爬取到第" + str(self.page_num) + "页后出错，错误信息为：\n" + str(s))
            self.save_data()

        # 数据保存

    def save_data(self):
        with open("data2.json", "a", encoding="utf8") as fp:
            fp.write(json.dumps(self.data, ensure_ascii=False, indent=4))
        print("数据保存成功")
        self.driver.close()


if __name__ == '__main__':
    # name = input("木托盘")
    # max_page = eval(input("10"))
    b2b = b2b("木方", 10)
    b2b.run()